package com.webmining.sherlock.crawler.parser;

import com.webmining.sherlock.crawler.CrawlingException;
import com.webmining.sherlock.crawler.Page;
import org.apache.log4j.Logger;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.cyberneko.html.parsers.DOMParser;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

public class HTMLParser {
	
	protected Document doc;
	
	protected Logger log;
	
	protected StringBuffer contents;

	protected StringBuffer summary;
	
	protected boolean contentCollect;
	
	protected Page page;
	
	public HTMLParser(Page page) {
		log = Logger.getLogger(HTMLParser.class);
		this.page = page;
	}
	
	public Document collect(InputStream in) throws CrawlingException {
		doc = new Document();
		// Assume contentCollect for documents without the BODY element
		contentCollect = true;
		contents = new StringBuffer();
		summary = new StringBuffer();
		try {
    		DOMParser htmlDOMParser = new DOMParser();
            BufferedReader r = new BufferedReader(
                    new InputStreamReader(in, "UTF-8"));
    		InputSource htmlIn = new InputSource(r);
    		htmlDOMParser.parse(htmlIn);
    		collect(htmlDOMParser.getDocument());
    		doc.add(new TextField("contents", contents.toString(), Field.Store.YES));
    		doc.add(new TextField("summary", summary.toString(), Field.Store.YES));
    		log.debug("summary:\n" + summary.toString());
    		log.debug("contents:\n" + contents.toString());
		} catch (SAXException se) {
			log.error("HTML parse exception");
			throw new CrawlingException("HTML parse exception");
		} catch (IOException ioe) {
			log.error("IO exception");
			throw new CrawlingException("IO exception");
		}
		return doc;
	}
	
	protected void collect(Node node) {
		int nodeType = node.getNodeType();
		// Actions for element nodes
		if (nodeType == Node.ELEMENT_NODE) {
			String nodeName = node.getNodeName();
			// Check if we are in the head element
			if("HEAD".equals(nodeName)) {
				contentCollect = false;
			}
			// Check if we are in the body element
			if("BODY".equals(nodeName)) {
				contentCollect = true;
			}
			// Get the document's title
			if ("TITLE".equals(nodeName)) {
				String title = getFirstChildText(node);
				if (title != null) {
					log.debug("Adding title '" + title + "'");
					doc.add(new TextField("title", title, Field.Store.YES));
				}
			}
			// Get any referenced link from this page
			if ("A".equals(nodeName)) {
				String link = getAttributeText(node, "href");
				if (link != null) {
					log.debug("Link found '" + link + "'");
					page.recordLink(link);
				}
			}
			if ("FRAME".equals(nodeName)) {
				String link = getAttributeText(node, "src");
				if (link != null) {
					log.debug("Link found '" + link + "'");
					page.recordLink(link);
				}
			}
		}
		// If a text is found and the contentCollect is active store it
		// in the contents buffer
		if (contentCollect && node.getNodeType() == Node.TEXT_NODE) {
			// Ensure is not a script text
			if (!"SCRIPT".equals(getParentNodeName(node))) {
				String text = node.getNodeValue();
				if (text != null) {
					text = clean(text);
					if (text.length() > 0) {
						contents.append(text);
						// Separe the text with a blank for not merge two
						// continues texts
						contents.append(" ");
						// TODO: remove static summary
						// Add the text to the summary
						if (summary.length() < 230) {
							summary.append(text.trim());
							summary.append(" ");
						}
					}
				}
			}
		}
		Node child = node.getFirstChild();
		while (child != null) {
			collect(child);
			child = child.getNextSibling();
		}
	}
	
	protected String clean(String text) {
		String cleanedText = text.replace('\r', ' ');
		cleanedText = cleanedText.replace('\n', ' ');
		cleanedText = cleanedText.replace('\t', ' ');
		return cleanedText.trim();
	}
	
	protected String getFirstChildText(Node node) {
		String text = null;
		Node child = node.getFirstChild();
		if (child != null && child.getNodeType() == Node.TEXT_NODE) {
			text = child.getNodeValue();
		}

        return text;
    }
	
	protected String getAttributeText(Node node, String attributeName) {
		String text = null;
		if (node.hasAttributes()) {
			Node attributeNode = node.getAttributes().getNamedItem(attributeName);
			if (attributeNode != null) {
				text = attributeNode.getNodeValue();
			}
		}
		return text;
	}
	
	protected String getParentNodeName(Node node) {
		String parentNodeName = null;
		Node parentNode = node.getParentNode();
		if (parentNode != null) {
			parentNodeName = parentNode.getNodeName();
		}
		return parentNodeName;
	}
	
}
